#include <fstream>
#include "DataMaker.h"
#include "service_log.hpp"
using namespace std;

DataMaker* DataMaker::m_pInstance = NULL;

int DataMaker::loadStopWord(const string& stopword_file, tr1::unordered_set<string>& sw_set)
{
    sw_set.clear();
    ifstream fin(stopword_file.c_str());
    if(!fin)
    {
        _ERROR("loadStopWord: \"%s\" can not open", stopword_file.c_str());
        return 0;
    }
    string line = "";
    while(getline(fin, line))
        sw_set.insert(line);

    return sw_set.size();
}

int DataMaker::loadSynWord(const string& synword_file, tr1::unordered_map<string, tr1::unordered_set<string> >& word_synset_map)
{
    word_synset_map.clear();
    ifstream fin(synword_file.c_str());
    if(!fin)
    {
        _ERROR("[loadSynWord: \"%s\" can not open]", synword_file.c_str());
        return 0;
    }
    string line = "";
    int c = 0;
    while(getline(fin, line))
    {
        c += 1;
        vector<string> v;
        m_st.split(line, "\t", v);
        if(v.size() < 2)
        {
            _INFO("[Warning in loadSynWord] [Format Error in %dth line]", c);
            continue;
        }
        word_synset_map[v[0]].insert(v[1]);
    }
    return word_synset_map.size();
}

bool DataMaker::make(const string& knowledge_file, 
        const string& stopword_file,
        const string& synword_file,
        const string& index_file,
        const string& findex_file)
{
    tr1::unordered_set<string> sw_set;
    loadStopWord(stopword_file, sw_set);

    tr1::unordered_map<string, tr1::unordered_set<string> > word_synset_map;
    loadSynWord(synword_file, word_synset_map);

    ifstream fin(knowledge_file.c_str());
    ofstream fout1(index_file.c_str());
    ofstream fout2(findex_file.c_str());

    if(!fin)
    {
        _ERROR("[DataMaker.make: \"%s\" can not open]", knowledge_file.c_str());
        return false;
    }

    string line = "";
    int c = 0;
    tr1::unordered_map<string, tr1::unordered_set<int> > key_index_map;
    while(getline(fin, line))
    {
        c += 1;
        vector<string> v;
        m_st.split(line, "\t", v);
        if(v.size() < 2)
        {
            _INFO("[Warning in reading \"%s\"] [Format Error in %dth line]", knowledge_file.c_str(), c);
            continue;
        }

        string question = v[0];
        string answer = v[1];

        fout2 << c << "\t" << question << "\t" << answer << endl;

        tr1::unordered_set<string> orig_set;
        orig_set.insert(question);
        for(tr1::unordered_map<string, tr1::unordered_set<string> >::iterator it = word_synset_map.begin(); it != word_synset_map.end(); ++ it)
        {
            if(question.find(it->first) != string::npos)
            {
                for(tr1::unordered_set<string>::iterator jt = it->second.begin(); jt != it->second.end(); ++jt)
                    orig_set.insert(*jt);
            }
        }


        for(tr1::unordered_set<string>::iterator it = orig_set.begin(); it != orig_set.end(); ++it)
        {
            vector<string> unigram_vec;
            vector<string> bigram_vec;

            m_seg.gbksegment2unigram(*it, unigram_vec);
            //for(int i = 0; i < unigram_vec.size(); ++i)
            //    cout << unigram_vec[i] << endl;
            //getchar();

            m_seg.unigram2bigram(unigram_vec, bigram_vec);

            //for(int i = 0; i < bigram_vec.size(); ++i)
            //    cout << bigram_vec[i] << endl;
            //getchar();

            for(int i = 0; i < unigram_vec.size(); ++i)
            {
                if(sw_set.find(unigram_vec[i]) == sw_set.end())
                    key_index_map[unigram_vec[i]].insert(c);
            }

            for(int i = 0; i < bigram_vec.size(); ++i)
            {
                if(sw_set.find(bigram_vec[i]) == sw_set.end())
                    key_index_map[bigram_vec[i]].insert(c);
            }
        }// for


    }// while

    cout << "key_index_map.size = " << key_index_map.size() << endl;

    for(tr1::unordered_map<string, tr1::unordered_set<int> >::iterator it = key_index_map.begin(); it != key_index_map.end(); ++it)
    {
        if(it->second.size() <= 0)
            continue;
        fout1 << it->first;
        for(tr1::unordered_set<int>::iterator jt = it->second.begin(); jt != it->second.end(); ++jt)
            fout1 << "\t" << *jt;
        fout1 << endl;
    }
    return true;
}
